package com.xmx.job.task;

import com.xmx.job.pojo.JobInfo;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

import java.util.Date;
import java.util.List;

/**
 * @Author Xumx
 * @Date 2021/3/19 14:01
 * @Version 1.0
 */
@Component
public class JobProcessor implements PageProcessor {

    private String url = "http://www.xsyu.edu.cn/xwzx/ssfc.htm";

    @Override
    public void process(Page page) {
        //解析页面，获取招聘信息详情的url地址
        //Html html1 = page.getHtml();
        List<Selectable> list = page.getHtml().css("div.whole-news ul.pieces-news li").nodes();

        //判断获取到的集合是否为空
        if (list.size()==0){
            //如果为空，表示这是招聘详情页,解析页面，获取招聘详情信息，保存数据
            this.saveJobInfo(page);

        }else {
            //如果不为空，表示这是列表页,解析出详情页的url地址，放到任务队列中
            for (Selectable selectable : list) {
                //获取url地址
                String jobInfoUrl = selectable.links().toString();
                //System.out.println(jobInfoUrl);
                page.addTargetRequest(jobInfoUrl);
            }

            //获取下一页的url
            String s = page.getHtml().css("div.joi a").nodes().get(0).links().toString();
            //System.out.println(s);
            //把url放到任务队列中
            page.addTargetRequest(s);
        }


        String html = page.getHtml().toString();
        System.out.println("爬取完成");
    }

    //解析页面，获取招聘详情信息，保存数据
    private void saveJobInfo(Page page) {
        //创建招聘详情对象
        JobInfo jobInfo = new JobInfo();

        //解析页面
        Html html = page.getHtml();

        //获取数据，封装到对象中
        jobInfo.setCompanyName(html.css("div.rbox h1","text").toString());
        jobInfo.setCompanyAddr(html.css("div.rbox div.time","text").toString());
        jobInfo.setCompanyInfo(html.css("div.rbox div.time","text").toString());
        jobInfo.setJobName(html.css("div.rbox div.time","text").toString());
        jobInfo.setJobAddr(html.css("div.rbox div.time","text").toString());
        jobInfo.setJobInfo(html.css("div.rbox div.time","text").toString());
        jobInfo.setUrl(page.getUrl().toString());
        jobInfo.setSalaryMin(1000);
        jobInfo.setSalaryMax(5000);
        jobInfo.setTime("123");

        //把结果保存起来
        page.putField("jobInfo",jobInfo);

    }

    private Site site = Site.me()
            .setCharset("utf8")        //设置编码
            .setTimeOut(10*1000)       //设置超时时间
            .setRetrySleepTime(3000)   //设置重试发间隔时间
            .setRetryTimes(3);         //设置重试的次数

    @Override
    public Site getSite() {
        return site;
    }

    @Autowired
    private SpringDataPipeline springDataPipeline;

    //initialDelay当任务启动后，等待多久执行方法
    //fixedDelay每隔多久执行方法
    @Scheduled(initialDelay = 1000,fixedDelay = 100*1000)
    public void process(){
        Spider.create(new JobProcessor())
                .addUrl(url)
                .setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(100000)))
                .thread(10)
                .addPipeline(this.springDataPipeline)
                .run();
    }
}
